mirror of
https://github.com/microsoft/vscode.git
synced 2026-10-28 13:26:52 +08:00
Engineering - fetch Chromium build scripts through the GitHub API with fallbacks (#344354)
* Engineering - fail and retry the Chromium clang script download setup-env.sh piped 'curl -s' of Chromium's tools/clang/scripts/update.py from raw.githubusercontent.com straight into python. When GitHub rate-limits the request (429), python runs the error page and the install fails with 'SyntaxError: illegal target for annotation': 15 Linux PR CI runs in 30 days, 10 of them on Oct 26. Download the script to a file with curl -f, retry transient errors, authenticate with GITHUB_TOKEN when it's set (all six callers set it) to get the higher rate limit, and only then run it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Engineering - fetch Chromium build scripts from Gitiles before GitHub raw.githubusercontent.com rate-limits shared agent IPs per network, so authenticated requests and short retries do not help during an outage. It failed the releasing insider 489293 (Linux ARMHF Prepare deb package, all five dpkg-shlibdeps.pl attempts got 429) and main CI 489263 (Linux X64 and Quality Install dependencies, update.py). Add build/lib/chromiumSource.ts, which downloads a file from the Chromium tree at the version pinned in cgmanifest.json, from Chromium's canonical Gitiles host first, then the GitHub mirror (authenticated, then anonymous), retrying the set after 10, 30 and 60 s, and only writes content that looks like a script. Use it for dpkg-shlibdeps.pl in calculate-deps.ts and for clang's update.py in setup-env.sh, which also drops setup-env.sh's second hard-coded copy of the Chromium version. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Bound each Chromium script download so a stalled source fails over Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Log which host served each Chromium script Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Try the GitHub API first: Gitiles answers 403 to ADO agents Validation 489314 showed chromium.googlesource.com returning 403 to every request from the ADO Linux agents, so it could not cover a raw GitHub 429. raw.githubusercontent.com limits per network even with a token, while the GitHub REST contents API limits authenticated requests per token. Try it first when a token is set, then raw (authenticated, anonymous), then Gitiles. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Bryan ChenandCopilot authored and GitHub committed
1 parent
225765a37b
commit
a58fdc7c54
4 files changed
+228
-40
No files matched your search
@@ -38,8 +38,13 @@ cat > "$HOME/.gyp/include.gypi" << 'EOF'
|
||||
EOF
|
||||
|
||||
if [ "$npm_config_arch" == "x64" ]; then
|
||||
# Download clang based on chromium revision used by vscode
|
||||
curl -s https://raw.githubusercontent.com/chromium/chromium/150.0.7871.250/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux
|
||||
# Download clang based on chromium revision used by vscode. Fetch Chromium's
|
||||
# update script to a file first, trying the GitHub API before the raw host,
|
||||
# which rate-limits shared agent IPs, and Chromium's Gitiles host.
|
||||
CLANG_UPDATE_PY="$(mktemp)"
|
||||
CLANG_UPDATE_PY="$CLANG_UPDATE_PY" node -e 'import { downloadChromiumFile } from "./build/lib/chromiumSource.ts"; downloadChromiumFile("tools/clang/scripts/update.py", process.env["CLANG_UPDATE_PY"]);'
|
||||
python "$CLANG_UPDATE_PY" --output-dir=$PWD/.build/CR_Clang --host-os=linux
|
||||
rm -f "$CLANG_UPDATE_PY"
|
||||
|
||||
# Download libcxx headers and objects from upstream electron releases
|
||||
DEBUG=libcxx-fetcher \
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
import { writeFileSync } from 'fs';
|
||||
import manifests from '../../cgmanifest.json' with { type: 'json' };
|
||||
|
||||
export type HttpGetResult = { readonly ok: true; readonly body: Buffer } | { readonly ok: false; readonly error: string };
|
||||
export type HttpGet = (url: string, headers: readonly string[]) => HttpGetResult;
|
||||
|
||||
interface ChromiumFileSource {
|
||||
readonly name: string;
|
||||
readonly url: string;
|
||||
readonly headers: readonly string[];
|
||||
decode(body: Buffer): Buffer;
|
||||
}
|
||||
|
||||
const DEFAULT_RETRY_DELAYS_MS = [10_000, 30_000, 60_000];
|
||||
|
||||
export function getChromiumVersion(): string {
|
||||
const chromiumManifest = manifests.registrations.find(registration => {
|
||||
return registration.component.type === 'git' && registration.component.git!.name === 'chromium';
|
||||
});
|
||||
if (!chromiumManifest?.version) {
|
||||
throw new Error('Cannot find the chromium registration version in cgmanifest.json');
|
||||
}
|
||||
return chromiumManifest.version;
|
||||
}
|
||||
|
||||
// Pass headers through a curl config on stdin so the token does not appear in the process list.
|
||||
// Bound each request so a stalled connection fails over to the next source instead of hanging.
|
||||
export function curlGet(url: string, headers: readonly string[]): HttpGetResult {
|
||||
const curlConfig = headers.map(header => `header = "${header}"\n`).join('');
|
||||
const result = spawnSync('curl', ['-fsSL', '--connect-timeout', '20', '--max-time', '120', '--config', '-', url], { input: curlConfig, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (result.status === 0) {
|
||||
return { ok: true, body: result.stdout };
|
||||
}
|
||||
return { ok: false, error: `curl failed (status ${result.status}, signal ${result.signal}${result.error ? `, error ${result.error.message}` : ''}): ${result.stderr?.toString('utf8').trim()}` };
|
||||
}
|
||||
|
||||
function getSources(filePath: string, version: string, token: string | undefined): ChromiumFileSource[] {
|
||||
const sources: ChromiumFileSource[] = [];
|
||||
const identity = (body: Buffer) => body;
|
||||
const rawUrl = `https://raw.githubusercontent.com/chromium/chromium/${version}/${filePath}`;
|
||||
if (token) {
|
||||
const authorization = `Authorization: Bearer ${token}`;
|
||||
// raw.githubusercontent.com rate-limits shared agent IPs per network, even with a token.
|
||||
// The REST API limits authenticated requests per token instead.
|
||||
sources.push({
|
||||
name: 'api.github.com',
|
||||
url: `https://api.github.com/repos/chromium/chromium/contents/${filePath}?ref=${version}`,
|
||||
headers: [authorization, 'Accept: application/vnd.github.raw+json', 'X-GitHub-Api-Version: 2022-11-28'],
|
||||
decode: identity,
|
||||
});
|
||||
sources.push({ name: 'raw.githubusercontent.com (authenticated)', url: rawUrl, headers: [authorization], decode: identity });
|
||||
}
|
||||
// An invalid token gets a 404 from GitHub, so also try anonymously.
|
||||
sources.push({ name: 'raw.githubusercontent.com', url: rawUrl, headers: [], decode: identity });
|
||||
// Chromium's canonical host last: it answers 403 from some cloud agent networks.
|
||||
// `format=TEXT` returns base64 content.
|
||||
sources.push({
|
||||
name: 'chromium.googlesource.com',
|
||||
url: `https://chromium.googlesource.com/chromium/src/+/refs/tags/${version}/${filePath}?format=TEXT`,
|
||||
headers: [],
|
||||
decode: body => Buffer.from(body.toString('ascii'), 'base64'),
|
||||
});
|
||||
return sources;
|
||||
}
|
||||
|
||||
export interface DownloadChromiumFileOptions {
|
||||
readonly version?: string;
|
||||
readonly token?: string;
|
||||
readonly httpGet?: HttpGet;
|
||||
readonly retryDelaysMs?: readonly number[];
|
||||
/** Rejects error pages served with a success status. Defaults to requiring a `#!` script header. */
|
||||
readonly validate?: (content: Buffer) => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a file from the Chromium source tree at the version pinned in cgmanifest.json, trying
|
||||
* the GitHub API, the GitHub raw host and Chromium's Gitiles host, and retrying the set with backoff.
|
||||
*/
|
||||
export function downloadChromiumFile(filePath: string, destination: string, options: DownloadChromiumFileOptions = {}): void {
|
||||
const version = options.version ?? getChromiumVersion();
|
||||
const httpGet = options.httpGet ?? curlGet;
|
||||
const retryDelaysMs = options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
|
||||
const validate = options.validate ?? (content => content.subarray(0, 2).toString('utf8') === '#!');
|
||||
const sources = getSources(filePath, version, options.token ?? process.env['GITHUB_TOKEN']);
|
||||
const errors: string[] = [];
|
||||
|
||||
for (let round = 0; round <= retryDelaysMs.length; round++) {
|
||||
for (const source of sources) {
|
||||
const result = httpGet(source.url, source.headers);
|
||||
let error: string;
|
||||
if (result.ok) {
|
||||
const content = source.decode(result.body);
|
||||
if (validate(content)) {
|
||||
writeFileSync(destination, content);
|
||||
console.log(`Downloaded Chromium ${version} ${filePath} from ${source.name}`);
|
||||
return;
|
||||
}
|
||||
error = 'downloaded content is not the expected file';
|
||||
} else {
|
||||
error = result.error;
|
||||
}
|
||||
const message = `Attempt ${round + 1}/${retryDelaysMs.length + 1} to download ${filePath} from ${source.name} failed: ${error}`;
|
||||
console.error(message);
|
||||
errors.push(message);
|
||||
}
|
||||
if (round < retryDelaysMs.length) {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, retryDelaysMs[round]);
|
||||
}
|
||||
}
|
||||
throw new Error(`Cannot download Chromium ${version} ${filePath}.\n${errors.join('\n')}`);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import assert from 'assert';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { suite, test } from 'node:test';
|
||||
import { downloadChromiumFile, getChromiumVersion, type HttpGetResult } from '../chromiumSource.ts';
|
||||
|
||||
const script = '#!/usr/bin/env python3\nprint("hello")\n';
|
||||
const rateLimited: HttpGetResult = { ok: false, error: 'curl failed (status 22): The requested URL returned error: 429' };
|
||||
|
||||
function withTempFile(fn: (destination: string) => void): void {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'chromium-source-test-'));
|
||||
try {
|
||||
fn(path.join(dir, 'file'));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
suite('chromiumSource', () => {
|
||||
test('reads the pinned version from cgmanifest.json', () => {
|
||||
assert.match(getChromiumVersion(), /^\d+\.\d+\.\d+\.\d+$/);
|
||||
});
|
||||
|
||||
test('downloads from the GitHub API with the token first', () => withTempFile(destination => {
|
||||
const requests: string[] = [];
|
||||
downloadChromiumFile('tools/clang/scripts/update.py', destination, {
|
||||
version: '1.2.3.4',
|
||||
token: 'secret',
|
||||
retryDelaysMs: [],
|
||||
httpGet: (url, headers) => {
|
||||
requests.push(`${url} ${headers.join('|')}`);
|
||||
return { ok: true, body: Buffer.from(script) };
|
||||
},
|
||||
});
|
||||
assert.deepStrictEqual(requests, ['https://api.github.com/repos/chromium/chromium/contents/tools/clang/scripts/update.py?ref=1.2.3.4 Authorization: Bearer secret|Accept: application/vnd.github.raw+json|X-GitHub-Api-Version: 2022-11-28']);
|
||||
assert.strictEqual(fs.readFileSync(destination, 'utf8'), script);
|
||||
}));
|
||||
|
||||
test('falls back to raw GitHub, then Gitiles, decoding its base64 body', () => withTempFile(destination => {
|
||||
const requests: string[] = [];
|
||||
downloadChromiumFile('a/b.pl', destination, {
|
||||
version: '1.2.3.4',
|
||||
token: 'secret',
|
||||
retryDelaysMs: [],
|
||||
httpGet: (url, headers) => {
|
||||
requests.push(`${new URL(url).host} ${headers.length ? headers[0] : '-'}`);
|
||||
return url.includes('googlesource') ? { ok: true, body: Buffer.from(Buffer.from(script).toString('base64')) } : rateLimited;
|
||||
},
|
||||
});
|
||||
assert.deepStrictEqual(requests, [
|
||||
'api.github.com Authorization: Bearer secret',
|
||||
'raw.githubusercontent.com Authorization: Bearer secret',
|
||||
'raw.githubusercontent.com -',
|
||||
'chromium.googlesource.com -',
|
||||
]);
|
||||
assert.strictEqual(fs.readFileSync(destination, 'utf8'), script);
|
||||
}));
|
||||
|
||||
test('skips the GitHub API without a token and rejects error pages served with a success status', () => withTempFile(destination => {
|
||||
const requests: string[] = [];
|
||||
downloadChromiumFile('a/b.pl', destination, {
|
||||
version: '1.2.3.4',
|
||||
token: '',
|
||||
retryDelaysMs: [0],
|
||||
httpGet: url => {
|
||||
requests.push(new URL(url).host);
|
||||
if (url.includes('githubusercontent')) {
|
||||
return requests.length === 3 ? { ok: true, body: Buffer.from(script) } : { ok: true, body: Buffer.from('429: Too Many Requests') };
|
||||
}
|
||||
return { ok: true, body: Buffer.from(Buffer.from('<!DOCTYPE html>').toString('base64')) };
|
||||
},
|
||||
});
|
||||
assert.deepStrictEqual(requests, ['raw.githubusercontent.com', 'chromium.googlesource.com', 'raw.githubusercontent.com']);
|
||||
assert.strictEqual(fs.readFileSync(destination, 'utf8'), script);
|
||||
}));
|
||||
|
||||
test('throws with every attempt once all rounds fail, and writes nothing', () => withTempFile(destination => {
|
||||
let calls = 0;
|
||||
assert.throws(() => downloadChromiumFile('a/b.pl', destination, {
|
||||
version: '1.2.3.4',
|
||||
token: 'secret',
|
||||
retryDelaysMs: [0, 0],
|
||||
httpGet: () => {
|
||||
calls++;
|
||||
return rateLimited;
|
||||
},
|
||||
}), (error: Error) => error.message.includes('Cannot download Chromium 1.2.3.4 a/b.pl') && error.message.includes('Attempt 3/3') && error.message.includes('api.github.com'));
|
||||
assert.strictEqual(calls, 12);
|
||||
assert.strictEqual(fs.existsSync(destination), false);
|
||||
}));
|
||||
});
|
||||
@@ -4,10 +4,10 @@
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
import { constants, mkdtempSync, readFileSync, rmSync, statSync } from 'fs';
|
||||
import { constants, mkdtempSync, statSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import path from 'path';
|
||||
import manifests from '../../../cgmanifest.json' with { type: 'json' };
|
||||
import { downloadChromiumFile } from '../../lib/chromiumSource.ts';
|
||||
import { additionalDeps } from './dep-lists.ts';
|
||||
import type { DebianArchString } from './types.ts';
|
||||
|
||||
@@ -22,43 +22,12 @@ let dpkgShlibdepsScript: string | undefined;
|
||||
|
||||
// Get the Chromium dpkg-shlibdeps file once per process.
|
||||
function getDpkgShlibdepsScript(): string {
|
||||
if (dpkgShlibdepsScript) {
|
||||
return dpkgShlibdepsScript;
|
||||
if (!dpkgShlibdepsScript) {
|
||||
const location = path.join(mkdtempSync(path.join(tmpdir(), 'dpkg-shlibdeps-')), 'dpkg-shlibdeps.pl');
|
||||
downloadChromiumFile('third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl', location);
|
||||
dpkgShlibdepsScript = location;
|
||||
}
|
||||
|
||||
const chromiumManifest = manifests.registrations.find(registration => {
|
||||
return registration.component.type === 'git' && registration.component.git!.name === 'chromium';
|
||||
});
|
||||
if (!chromiumManifest) {
|
||||
throw new Error('Cannot find the chromium registration in cgmanifest.json');
|
||||
}
|
||||
const dpkgShlibdepsUrl = `https://raw.githubusercontent.com/chromium/chromium/${chromiumManifest.version}/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl`;
|
||||
const dpkgShlibdepsScriptLocation = path.join(mkdtempSync(path.join(tmpdir(), 'dpkg-shlibdeps-')), 'dpkg-shlibdeps.pl');
|
||||
|
||||
// Pass the token through a curl config on stdin so it does not appear in the process list.
|
||||
// Authenticated requests avoid the anonymous rate limit on shared agent IPs, but an invalid
|
||||
// token gets a 404, so alternate with anonymous attempts.
|
||||
const token = process.env['GITHUB_TOKEN'];
|
||||
const maxAttempts = 5;
|
||||
let lastError = '';
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
const useToken = !!token && attempt % 2 === 1;
|
||||
const curlConfig = useToken ? `header = "Authorization: Bearer ${token}"\n` : '';
|
||||
rmSync(dpkgShlibdepsScriptLocation, { force: true });
|
||||
const result = spawnSync('curl', ['-fsSL', '--config', '-', '-o', dpkgShlibdepsScriptLocation, dpkgShlibdepsUrl], { input: curlConfig });
|
||||
if (result.status === 0 && readFileSync(dpkgShlibdepsScriptLocation, 'utf8').startsWith('#!')) {
|
||||
dpkgShlibdepsScript = dpkgShlibdepsScriptLocation;
|
||||
return dpkgShlibdepsScript;
|
||||
}
|
||||
lastError = result.status === 0
|
||||
? 'Downloaded file is not a Perl script'
|
||||
: `curl failed (status ${result.status}, signal ${result.signal}${result.error ? `, error ${result.error.message}` : ''}). Stderr:\n${result.stderr?.toString('utf8')}`;
|
||||
console.error(`Attempt ${attempt}/${maxAttempts} (${useToken ? 'authenticated' : 'anonymous'}) to retrieve dpkg-shlibdeps failed: ${lastError}`);
|
||||
if (attempt < maxAttempts) {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, attempt * 5000);
|
||||
}
|
||||
}
|
||||
throw new Error(`Cannot retrieve dpkg-shlibdeps from ${dpkgShlibdepsUrl}. ${lastError}`);
|
||||
return dpkgShlibdepsScript;
|
||||
}
|
||||
|
||||
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
|
||||
|
||||
Reference in new issue
Block a user