Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
ulixee
GitHub Repository: ulixee/secret-agent
 Path: blob/main/replay/backend/api/ReplayResources.ts
1030 views
1
import { ProtocolResponse } from 'electron';

2
import * as Http from 'http';

3
import { Readable } from 'stream';

4
import getResolvable from '~shared/utils/promise';

5
import { IReplayResource } from '~shared/interfaces/IReplayResource';

6
import decompress from '~shared/utils/decompress';

7


8
const packageJson = require('../../package.json');

9


10
type IResolvable<T> = {

11
  resolve: (resource: T) => void;

12
  promise: Promise<T>;

13
};

14
const resourceWhitelist = new Set<string>([

15
  'Ico',

16
  'Image',

17
  'Media',

18
  'Font',

19
  'Stylesheet',

20
  'Other',

21
  'Document',

22
]);

23


24
export default class ReplayResources {

25
  private resourcesByUrl = new Map<string, IResolvable<IReplayResource>>();

26
  private resourcesById = new Map<number, IReplayResource>();

27
  private resourceBodyById = new Map<number, Promise<ProtocolResponse>>();

28


29
  public onResource(resourceMeta: IReplayResource): void {

30
    const { url } = resourceMeta;

31
    this.initResource(url);

32


33
    this.resourcesById.set(resourceMeta.id, resourceMeta);

34
    this.resourcesByUrl.get(url).resolve(resourceMeta);

35
  }

36


37
  public async get(urlStr: string): Promise<IReplayResource> {

38
    const url = urlStr.split('#').shift();

39
    this.initResource(url);

40
    return await this.resourcesByUrl.get(url).promise;

41
  }

42


43
  public async getContent(

44
    resourceId: number,

45
    baseHost: string,

46
    dataDir: string,

47
  ): Promise<ProtocolResponse> {

48
    const resource = this.resourcesById.get(resourceId);

49
    if (!resourceWhitelist.has(resource.type)) {

50
      console.log('skipping resource', resource);

51


52
      return <ProtocolResponse>{

53
        data: Readable.from([]),

54
        statusCode: 404,

55
      };

56
    }

57
    if (!this.resourceBodyById.has(resourceId)) {

58
      const { resolve, reject, promise } = getResolvable<ProtocolResponse>();

59
      this.resourceBodyById.set(resourceId, promise);

60
      const reqHeaders = { headers: { 'x-data-location': dataDir } };

61
      const req = Http.get(`${baseHost}/resource/${resourceId}`, reqHeaders, async res => {

62
        res.on('error', reject);

63


64
        const contentType = res.headers['content-type'];

65
        const encoding = res.headers['content-encoding'];

66
        const headers: any = {

67
          'Cache-Control': 'public, max-age=604800, immutable',

68
          'Content-Type': contentType,

69
          'X-Replay-Agent': `Secret Agent Replay v${packageJson.version}`,

70
        };

71
        if (res.headers.location) {

72
          headers.location = res.headers.location;

73
        }

74


75
        const buffer: Buffer[] = [];

76
        for await (const chunk of res) {

77
          buffer.push(chunk);

78
        }

79


80
        let body = Buffer.concat(buffer);

81


82
        if (encoding) {

83
          body = await decompress(body, encoding);

84
        }

85


86
        if (resource.type === 'Document' && !isAllowedDocumentContentType(contentType)) {

87
          const first100 = body.slice(0, 200).toString();

88


89
          let doctype = '';

90
          const match = first100.match(/^\s*<!DOCTYPE([^>]*?)>/i);

91
          if (match) {

92
            doctype = `<!DOCTYPE${match[1] ?? ''}>\n`;

93
          }

94


95
          body = Buffer.from(`${doctype}<html><head></head><body></body></html>`);

96
        }

97


98
        resolve(<ProtocolResponse>{

99
          data: body,

100
          headers,

101
          statusCode: res.statusCode,

102
        });

103
      });

104
      req.on('error', reject);

105
      req.end();

106
    }

107
    const cached = await this.resourceBodyById.get(resourceId);

108
    return {

109
      ...cached,

110
      data: Readable.from(cached.data),

111
    };

112
  }

113


114
  private initResource(url: string) {

115
    if (!this.resourcesByUrl.has(url)) {

116
      this.resourcesByUrl.set(url, getResolvable<IReplayResource>());

117
    }

118
  }

119
}

120


121
function isAllowedDocumentContentType(contentType: string) {

122
  if (!contentType) return false;

123
  return (

124
    contentType.includes('json') || contentType.includes('svg') || contentType.includes('font')

125
  );

126
}

127


128