CoCalc -- product-build-linux-compile.yml

GitHub Repository: microsoft/vscode
Path: blob/main/build/azure-pipelines/linux/steps/product-build-linux-compile.yml
⁴⁷⁷⁴ views
1
parameters:
2
  - name: VSCODE_ARCH
3
    type: string
4
  - name: VSCODE_CIBUILD
5
    type: boolean
6
  - name: VSCODE_QUALITY
7
    type: string
8
  - name: VSCODE_BUILD_LINUX_SNAP
9
    type: boolean
10
    default: false
11
  - name: VSCODE_RUN_ELECTRON_TESTS
12
    type: boolean
13
    default: false
14
  - name: VSCODE_RUN_BROWSER_TESTS
15
    type: boolean
16
    default: false
17
  - name: VSCODE_RUN_REMOTE_TESTS
18
    type: boolean
19
    default: false
20

21
steps:
22
  - template: ../../common/checkout.yml@self
23

24
  - task: NodeTool@0
25
    inputs:
26
      versionSource: fromFile
27
      versionFilePath: .nvmrc
28

29
  - template: ../../distro/download-distro.yml@self
30

31
  - task: AzureKeyVault@2
32
    displayName: "Azure Key Vault: Get Secrets"
33
    inputs:
34
      azureSubscription: vscode
35
      KeyVaultName: vscode-build-secrets
36
      SecretsFilter: "github-distro-mixin-password"
37

38
  - task: DownloadPipelineArtifact@2
39
    inputs:
40
      artifact: Compilation
41
      path: $(Build.ArtifactStagingDirectory)
42
    displayName: Download compilation output
43

44
  - script: tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz
45
    displayName: Extract compilation output
46

47
  - script: |
48
      set -e
49
      # Start X server
50
      ./build/azure-pipelines/linux/apt-retry.sh sudo apt-get update
51
      ./build/azure-pipelines/linux/apt-retry.sh sudo apt-get install -y pkg-config \
52
        xvfb \
53
        libgtk-3-0 \
54
        libxkbfile-dev \
55
        libkrb5-dev \
56
        libgbm1 \
57
        rpm
58
      sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb
59
      sudo chmod +x /etc/init.d/xvfb
60
      sudo update-rc.d xvfb defaults
61
      sudo service xvfb start
62
    displayName: Setup system services
63

64
  - script: node build/setup-npm-registry.ts $NPM_REGISTRY
65
    condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none'))
66
    displayName: Setup NPM Registry
67

68
  - script: mkdir -p .build && node build/azure-pipelines/common/computeNodeModulesCacheKey.ts linux $VSCODE_ARCH $(node -p process.arch) > .build/packagelockhash
69
    displayName: Prepare node_modules cache key
70

71
  - task: Cache@2
72
    inputs:
73
      key: '"node_modules" | .build/packagelockhash'
74
      path: .build/node_modules_cache
75
      cacheHitVar: NODE_MODULES_RESTORED
76
    displayName: Restore node_modules cache
77

78
  - script: tar -xzf .build/node_modules_cache/cache.tgz
79
    condition: and(succeeded(), eq(variables.NODE_MODULES_RESTORED, 'true'))
80
    displayName: Extract node_modules cache
81

82
  - script: |
83
      set -e
84
      # Set the private NPM registry to the global npmrc file
85
      # so that authentication works for subfolders like build/, remote/, extensions/ etc
86
      # which does not have their own .npmrc file
87
      npm config set registry "$NPM_REGISTRY"
88
      echo "##vso[task.setvariable variable=NPMRC_PATH]$(npm config get userconfig)"
89
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none'))
90
    displayName: Setup NPM
91

92
  - task: npmAuthenticate@0
93
    inputs:
94
      workingFile: $(NPMRC_PATH)
95
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none'))
96
    displayName: Setup NPM Authentication
97

98
  - script: |
99
      set -e
100

101
      for i in {1..5}; do # try 5 times
102
        npm ci && break
103
        if [ $i -eq 5 ]; then
104
          echo "Npm install failed too many times" >&2
105
          exit 1
106
        fi
107
        echo "Npm install failed $i, trying again..."
108
      done
109
    workingDirectory: build
110
    env:
111
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
112
    displayName: Install build dependencies
113
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
114

115
  # Step will be used by both verify glibcxx version for remote server and building rpm package,
116
  # hence avoid adding it behind NODE_MODULES_RESTORED condition.
117
  - script: |
118
      set -e
119
      SYSROOT_ARCH=$VSCODE_ARCH
120
      if [ "$SYSROOT_ARCH" == "x64" ]; then
121
        SYSROOT_ARCH="amd64"
122
      fi
123
      export VSCODE_SYSROOT_DIR=$(Build.SourcesDirectory)/.build/sysroots/glibc-2.28-gcc-8.5.0
124
      SYSROOT_ARCH="$SYSROOT_ARCH" VSCODE_SYSROOT_PREFIX="-glibc-2.28-gcc-8.5.0" node -e 'import { getVSCodeSysroot } from "./build/linux/debian/install-sysroot.ts"; (async () => { await getVSCodeSysroot(process.env["SYSROOT_ARCH"]); })()'
125
    env:
126
      VSCODE_ARCH: $(VSCODE_ARCH)
127
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
128
    displayName: Download vscode sysroots
129

130
  - script: |
131
      set -e
132

133
      source ./build/azure-pipelines/linux/setup-env.sh
134

135
      # Run preinstall script before root dependencies are installed
136
      # so that v8 headers are patched correctly for native modules.
137
      node build/npm/preinstall.ts
138

139
      for i in {1..5}; do # try 5 times
140
        npm ci && break
141
        if [ $i -eq 5 ]; then
142
          echo "Npm install failed too many times" >&2
143
          exit 1
144
        fi
145
        echo "Npm install failed $i, trying again..."
146
      done
147
    env:
148
      npm_config_arch: $(NPM_ARCH)
149
      VSCODE_ARCH: $(VSCODE_ARCH)
150
      ELECTRON_SKIP_BINARY_DOWNLOAD: 1
151
      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
152
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
153
    displayName: Install dependencies
154
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
155

156
  - script: node build/azure-pipelines/distro/mixin-npm.ts
157
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
158
    displayName: Mixin distro node modules
159

160
  - script: |
161
      set -e
162
      node build/azure-pipelines/common/listNodeModules.ts .build/node_modules_list.txt
163
      mkdir -p .build/node_modules_cache
164
      tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt
165
    condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'))
166
    displayName: Create node_modules archive
167

168
  - script: node build/azure-pipelines/distro/mixin-quality.ts
169
    displayName: Mixin distro quality
170

171
  - template: ../../common/install-builtin-extensions.yml@self
172

173
  - ${{ if ne(parameters.VSCODE_CIBUILD, true) }}:
174
    - script: npm run copy-policy-dto --prefix build && node build/lib/policies/policyGenerator.ts build/lib/policies/policyData.jsonc linux
175
      displayName: Generate policy definitions
176
      retryCountOnTaskFailure: 3
177

178
  - script: |
179
      set -e
180
      npm run gulp vscode-linux-$(VSCODE_ARCH)-min-ci
181
      ARCHIVE_PATH=".build/linux/client/code-${{ parameters.VSCODE_QUALITY }}-$(VSCODE_ARCH)-$(date +%s).tar.gz"
182
      mkdir -p $(dirname $ARCHIVE_PATH)
183
      echo "##vso[task.setvariable variable=CLIENT_PATH]$ARCHIVE_PATH"
184
      echo "##vso[task.setvariable variable=CLIENT_ARCHIVE_NAME]$(basename $ARCHIVE_PATH)"
185
    env:
186
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
187
    displayName: Build client
188

189
  - ${{ if ne(parameters.VSCODE_CIBUILD, true) }}:
190
    - task: DownloadPipelineArtifact@2
191
      inputs:
192
        artifact: $(ARTIFACT_PREFIX)vscode_cli_linux_$(VSCODE_ARCH)_cli
193
        patterns: "**"
194
        path: $(Build.ArtifactStagingDirectory)/cli
195
      displayName: Download VS Code CLI
196

197
    - script: |
198
        set -e
199
        tar -xzvf $(Build.ArtifactStagingDirectory)/cli/*.tar.gz -C $(Build.ArtifactStagingDirectory)/cli
200
        CLI_APP_NAME=$(node -p "require(\"$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)/resources/app/product.json\").tunnelApplicationName")
201
        APP_NAME=$(node -p "require(\"$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)/resources/app/product.json\").applicationName")
202
        mv $(Build.ArtifactStagingDirectory)/cli/$APP_NAME $(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)/bin/$CLI_APP_NAME
203
      displayName: Mix in CLI
204

205
  - script: |
206
      set -e
207
      tar -czf $CLIENT_PATH -C .. VSCode-linux-$(VSCODE_ARCH)
208
    env:
209
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
210
    displayName: Archive client
211

212
  - script: |
213
      set -e
214
      npm run gulp vscode-reh-linux-$(VSCODE_ARCH)-min-ci
215
      mv ../vscode-reh-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH) # TODO@joaomoreno
216
      ARCHIVE_PATH=".build/linux/server/vscode-server-linux-$(VSCODE_ARCH).tar.gz"
217
      UNARCHIVE_PATH="`pwd`/../vscode-server-linux-$(VSCODE_ARCH)"
218
      mkdir -p $(dirname $ARCHIVE_PATH)
219
      tar --owner=0 --group=0 -czf $ARCHIVE_PATH -C .. vscode-server-linux-$(VSCODE_ARCH)
220
      echo "##vso[task.setvariable variable=SERVER_PATH]$ARCHIVE_PATH"
221
      echo "##vso[task.setvariable variable=SERVER_UNARCHIVE_PATH]$UNARCHIVE_PATH"
222
    env:
223
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
224
    displayName: Build server
225

226
  - script: |
227
      set -e
228
      npm run gulp vscode-reh-web-linux-$(VSCODE_ARCH)-min-ci
229
      mv ../vscode-reh-web-linux-$(VSCODE_ARCH) ../vscode-server-linux-$(VSCODE_ARCH)-web # TODO@joaomoreno
230
      ARCHIVE_PATH=".build/linux/web/vscode-server-linux-$(VSCODE_ARCH)-web.tar.gz"
231
      mkdir -p $(dirname $ARCHIVE_PATH)
232
      tar --owner=0 --group=0 -czf $ARCHIVE_PATH -C .. vscode-server-linux-$(VSCODE_ARCH)-web
233
      echo "##vso[task.setvariable variable=WEB_PATH]$ARCHIVE_PATH"
234
    env:
235
      GITHUB_TOKEN: "$(github-distro-mixin-password)"
236
    displayName: Build server (web)
237

238
  - ${{ if or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64')) }}:
239
    - script: |
240
        set -e
241

242
        EXPECTED_GLIBC_VERSION="2.28" \
243
        EXPECTED_GLIBCXX_VERSION="3.4.25" \
244
        VSCODE_SYSROOT_DIR="$(Build.SourcesDirectory)/.build/sysroots/glibc-2.28-gcc-8.5.0" \
245
        ./build/azure-pipelines/linux/verify-glibc-requirements.sh
246
      env:
247
        SEARCH_PATH: $(SERVER_UNARCHIVE_PATH)
248
        npm_config_arch: $(NPM_ARCH)
249
        VSCODE_ARCH: $(VSCODE_ARCH)
250
      displayName: Check GLIBC and GLIBCXX dependencies in server archive
251

252
  - ${{ else }}:
253
    - script: |
254
        set -e
255

256
        EXPECTED_GLIBC_VERSION="2.28" \
257
        EXPECTED_GLIBCXX_VERSION="3.4.26" \
258
        VSCODE_SYSROOT_DIR="$(Build.SourcesDirectory)/.build/sysroots/glibc-2.28-gcc-8.5.0" \
259
        ./build/azure-pipelines/linux/verify-glibc-requirements.sh
260
      env:
261
        SEARCH_PATH: $(SERVER_UNARCHIVE_PATH)
262
        npm_config_arch: $(NPM_ARCH)
263
        VSCODE_ARCH: $(VSCODE_ARCH)
264
      displayName: Check GLIBC and GLIBCXX dependencies in server archive
265

266
  - ${{ if ne(parameters.VSCODE_CIBUILD, true) }}:
267
    - script:  |
268
        set -e
269
        npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-deb"
270
      env:
271
        GITHUB_TOKEN: "$(github-distro-mixin-password)"
272
      displayName: Prepare deb package
273

274
    - script:  |
275
        set -e
276
        npm run gulp "vscode-linux-$(VSCODE_ARCH)-build-deb"
277
        mkdir -p .build/linux/deb
278
        cp .build/linux/deb/*/deb/*.deb .build/linux/deb/
279
        file_output=$(file $(ls .build/linux/deb/*.deb))
280
        if [[ "$file_output" != *"data compression xz"* ]]; then
281
          echo "Error: unknown compression. $file_output"
282
          exit 1
283
        fi
284
        echo "##vso[task.setvariable variable=DEB_PATH]$(ls .build/linux/deb/*.deb)"
285
        echo "##vso[task.setvariable variable=DEB_PACKAGE_NAME]$(basename $(ls .build/linux/deb/*.deb))"
286
      displayName: Build deb package
287

288
    - script:  |
289
        set -e
290
        TRIPLE=""
291
        if [ "$VSCODE_ARCH" == "x64" ]; then
292
          TRIPLE="x86_64-linux-gnu"
293
        elif [ "$VSCODE_ARCH" == "arm64" ]; then
294
          TRIPLE="aarch64-linux-gnu"
295
        elif [ "$VSCODE_ARCH" == "armhf" ]; then
296
          TRIPLE="arm-rpi-linux-gnueabihf"
297
        fi
298
        export VSCODE_SYSROOT_DIR=$(Build.SourcesDirectory)/.build/sysroots/glibc-2.28-gcc-10.5.0
299
        export STRIP="$VSCODE_SYSROOT_DIR/$TRIPLE/$TRIPLE/bin/strip"
300
        npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-rpm"
301
      env:
302
        VSCODE_ARCH: $(VSCODE_ARCH)
303
      displayName: Prepare rpm package
304

305
    - script:  |
306
        set -e
307
        npm run gulp "vscode-linux-$(VSCODE_ARCH)-build-rpm"
308
        mkdir -p .build/linux/rpm
309
        cp .build/linux/rpm/*/*.rpm .build/linux/rpm/
310
        echo "##vso[task.setvariable variable=RPM_PATH]$(ls .build/linux/rpm/*.rpm)"
311
        echo "##vso[task.setvariable variable=RPM_PACKAGE_NAME]$(basename $(ls .build/linux/rpm/*.rpm))"
312
      displayName: Build rpm package
313

314
    - ${{ if eq(parameters.VSCODE_ARCH, 'x64') }}:
315
      - task: Docker@1
316
        inputs:
317
          azureSubscriptionEndpoint: vscode
318
          azureContainerRegistry: vscodehub.azurecr.io
319
          command: login
320
        displayName: Login to Container Registry
321

322
      - ${{ if eq(parameters.VSCODE_BUILD_LINUX_SNAP, true) }}:
323
        - script: |
324
            set -e
325
            npm run gulp "vscode-linux-$(VSCODE_ARCH)-prepare-snap"
326
            sudo -E docker run -e VSCODE_ARCH -e VSCODE_QUALITY -v $(pwd):/work -w /work vscodehub.azurecr.io/vscode-linux-build-agent:snapcraft-x64 /bin/bash -c "./build/azure-pipelines/linux/build-snap.sh"
327

328
            SNAP_ROOT="$(pwd)/.build/linux/snap/$(VSCODE_ARCH)"
329
            SNAP_EXTRACTED_PATH=$(find $SNAP_ROOT -maxdepth 1 -type d -name 'code-*')
330

331
            mkdir -p .build/linux/snap
332
            cp $(find $SNAP_ROOT -maxdepth 1 -type f -name '*.snap') .build/linux/snap/
333

334
            # SBOM tool doesn't like recursive symlinks
335
            sudo find $SNAP_EXTRACTED_PATH -type l -delete
336

337
            echo "##vso[task.setvariable variable=SNAP_EXTRACTED_PATH]$SNAP_EXTRACTED_PATH"
338
            echo "##vso[task.setvariable variable=SNAP_PATH]$(ls .build/linux/snap/*.snap)"
339
            echo "##vso[task.setvariable variable=SNAP_PACKAGE_NAME]$(basename $(ls .build/linux/snap/*.snap))"
340
          env:
341
            VSCODE_ARCH: $(VSCODE_ARCH)
342
          displayName: Build snap package
343

344
    - task: UseDotNet@2
345
      inputs:
346
        version: 6.x
347

348
    - task: EsrpCodeSigning@5
349
      inputs:
350
        UseMSIAuthentication: true
351
        ConnectedServiceName: vscode-esrp
352
        AppRegistrationClientId: $(ESRP_CLIENT_ID)
353
        AppRegistrationTenantId: $(ESRP_TENANT_ID)
354
        AuthAKVName: vscode-esrp
355
        AuthSignCertName: esrp-sign
356
        FolderPath: .
357
        Pattern: noop
358
      displayName: 'Install ESRP Tooling'
359

360
    - pwsh: |
361
        . build/azure-pipelines/win32/exec.ps1
362
        $ErrorActionPreference = "Stop"
363
        $EsrpCodeSigningTool = (gci -directory -filter EsrpCodeSigning_* $(Agent.RootDirectory)/_tasks | Select-Object -last 1).FullName
364
        $Version = (gci -directory $EsrpCodeSigningTool | Select-Object -last 1).FullName
365
        echo "##vso[task.setvariable variable=EsrpCliDllPath]$Version/net6.0/esrpcli.dll"
366
      displayName: Find ESRP CLI
367

368
    - script: npx deemon --detach --wait node build/azure-pipelines/linux/codesign.ts
369
      env:
370
        EsrpCliDllPath: $(EsrpCliDllPath)
371
        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
372
      displayName: ✍️ Codesign deb & rpm
373

374
  - ${{ if or(eq(parameters.VSCODE_RUN_ELECTRON_TESTS, true), eq(parameters.VSCODE_RUN_BROWSER_TESTS, true), eq(parameters.VSCODE_RUN_REMOTE_TESTS, true)) }}:
375
    - template: product-build-linux-test.yml@self
376
      parameters:
377
        VSCODE_RUN_ELECTRON_TESTS: ${{ parameters.VSCODE_RUN_ELECTRON_TESTS }}
378
        VSCODE_RUN_BROWSER_TESTS: ${{ parameters.VSCODE_RUN_BROWSER_TESTS }}
379
        VSCODE_RUN_REMOTE_TESTS: ${{ parameters.VSCODE_RUN_REMOTE_TESTS }}
380

381
  - ${{ if ne(parameters.VSCODE_CIBUILD, true) }}:
382
    - script: npx deemon --attach node build/azure-pipelines/linux/codesign.ts
383
      condition: succeededOrFailed()
384
      displayName: "✍️ Post-job: Codesign deb & rpm"
385

386
    - script: |
387
        set -e
388

389
        mkdir -p $(Build.ArtifactStagingDirectory)/out/client
390
        mv $(CLIENT_PATH) $(Build.ArtifactStagingDirectory)/out/client/$(CLIENT_ARCHIVE_NAME)
391

392
        mkdir -p $(Build.ArtifactStagingDirectory)/out/server
393
        mv $(SERVER_PATH) $(Build.ArtifactStagingDirectory)/out/server/$(basename $(SERVER_PATH))
394

395
        mkdir -p $(Build.ArtifactStagingDirectory)/out/web
396
        mv $(WEB_PATH) $(Build.ArtifactStagingDirectory)/out/web/$(basename $(WEB_PATH))
397

398
        mkdir -p $(Build.ArtifactStagingDirectory)/out/deb
399
        mv $(DEB_PATH) $(Build.ArtifactStagingDirectory)/out/deb/$(DEB_PACKAGE_NAME)
400

401
        mkdir -p $(Build.ArtifactStagingDirectory)/out/rpm
402
        mv $(RPM_PATH) $(Build.ArtifactStagingDirectory)/out/rpm/$(RPM_PACKAGE_NAME)
403

404
        if [ -n "$SNAP_PATH" ]; then
405
          mkdir -p $(Build.ArtifactStagingDirectory)/out/snap
406
          mv $(SNAP_PATH) $(Build.ArtifactStagingDirectory)/out/snap/$(SNAP_PACKAGE_NAME)
407
        fi
408

409
        # SBOM generation uses hard links which are not supported by the Linux kernel
410
        # for files that have the SUID bit set, so we need to remove the SUID bit from
411
        # the chrome-sandbox file.
412
        sudo ls -l $(Agent.BuildDirectory)/VSCode-linux-$(VSCODE_ARCH)/chrome-sandbox
413
        sudo chmod u-s $(Agent.BuildDirectory)/VSCode-linux-$(VSCODE_ARCH)/chrome-sandbox
414
        sudo ls -l $(Agent.BuildDirectory)/VSCode-linux-$(VSCODE_ARCH)/chrome-sandbox
415
      displayName: Move artifacts to out directory
416

417
Product

Resources

Company
