fetch-native.mjs

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *--------------------------------------------------------------------------------------------*/

/**
 * Downloads the native runtime artifacts for one platform classifier.
 *
 * Steps:
 *   1. Read the pinned version and the SHA-512 `integrity` value for
 *      `@github/copilot-<classifier>` from `nodejs/package-lock.json`.
 *   2. `npm pack` that exact version into the staging directory.
 *   3. Verify the downloaded tarball against the `integrity` value.
 *   4. Stage the hostless runtime tree, flattening the selected prebuild directory
 *      beside the package's retained top-level runtime assets.
 *   5. Write an inventory consumed by the SDK's generic classpath extractor.
 *   6. Write `<staging>/<classifier>/native/<classifier>/platform.properties`.
 *
 * Usage: node fetch-native.mjs <repoRoot> <stagingDir> <classifier>
 */

import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';

const excludedTopLevel = new Set([
  'app.js',
  'assets',
  'changelog.json',
  'copilot',
  'copilot.exe',
  'copilot-sdk',
  'foundry-local-sdk',
  'index.js',
  'LICENSE.md',
  'napi-oop-runtime',
  'npm-loader.js',
  'package.json',
  'preloads',
  'pvrecorder',
  'queries',
  'README.md',
  'sdk',
  'sea-loader.js',
  'webview',
]);

const [repoRoot, stagingDir, classifier] = process.argv.slice(2);

if (!repoRoot || !stagingDir || !classifier) {
  console.error('Usage: node fetch-native.mjs <repoRoot> <stagingDir> <classifier>');
  process.exit(1);
}

const lockPath = path.join(repoRoot, 'nodejs', 'package-lock.json');
const packageName = `@github/copilot-${classifier}`;
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
const entry = lock.packages?.[`node_modules/${packageName}`];

if (!entry?.version || !entry?.integrity) {
  console.error(`Could not find version/integrity for ${packageName} in ${lockPath}`);
  process.exit(1);
}

const { version, integrity } = entry;
if (!integrity.startsWith('sha512-')) {
  console.error(`Unsupported integrity algorithm for ${packageName}: ${integrity}`);
  process.exit(1);
}

const outDir = path.join(stagingDir, classifier);
const resourceDir = path.join(outDir, 'native', classifier);
const runtimePath = path.join(resourceDir, 'runtime.node');
const isWindows = classifier.startsWith('win32');
const wrapperFilename = isWindows ? 'copilot-runtime.exe' : 'copilot-runtime';
const wrapperPath = path.join(resourceDir, wrapperFilename);
const inventoryPath = path.join(resourceDir, 'runtime-assets.list');
const platformPropertiesPath = path.join(resourceDir, 'platform.properties');
const expectedPlatformProperties = `classifier=${classifier}\nversion=${version}\n`;
const stagingSchema = 'hostless-runtime-v2';
const stampPath = path.join(outDir, '.version');

// Idempotence: skip the download only when every required staged artifact
// matches the package identity recorded in the stamp.
if (
  fs.existsSync(runtimePath) &&
  fs.existsSync(wrapperPath) &&
  fs.existsSync(inventoryPath) &&
  fs.existsSync(platformPropertiesPath) &&
  fs.existsSync(stampPath)
) {
  const stampLines = fs.readFileSync(stampPath, 'utf8').trim().split('\n');
  const stampSchema = stampLines[0] || '';
  const stampVersion = stampLines[1] || '';
  const stampIntegrity = stampLines[2] || '';
  const stampTreeDigest = stampLines[3] || '';
  const currentTreeDigest = digestTree(resourceDir);
  const currentPlatformProperties = fs.readFileSync(platformPropertiesPath, 'utf8');
  if (
    stampSchema === stagingSchema &&
    stampVersion === version &&
    stampIntegrity === integrity &&
    stampTreeDigest === currentTreeDigest &&
    currentPlatformProperties === expectedPlatformProperties
  ) {
    console.log(`${packageName}@${version} already staged at ${runtimePath}`);
    process.exit(0);
  }
}

fs.rmSync(outDir, { recursive: true, force: true });
fs.mkdirSync(resourceDir, { recursive: true });

console.log(`Downloading ${packageName}@${version} ...`);
const packOutput = execFileSync('npm', ['pack', `${packageName}@${version}`, '--pack-destination', outDir], {
  encoding: 'utf8',
  shell: process.platform === 'win32',
});
const tarballName = packOutput.trim().split('\n').pop().trim();
const tarballPath = path.join(outDir, tarballName);

const actual = `sha512-${createHash('sha512').update(fs.readFileSync(tarballPath)).digest('base64')}`;
if (actual !== integrity) {
  console.error(`Integrity verification failed for ${tarballPath}`);
  console.error(`  expected: ${integrity}`);
  console.error(`  actual:   ${actual}`);
  process.exit(1);
}
console.log(`Integrity verified (${integrity.slice(0, 20)}...).`);

const inventory = [];
const members = execFileSync('tar', ['-tzf', tarballPath], { encoding: 'utf8' })
  .split(/\r?\n/)
  .filter(Boolean);
for (const member of members) {
  const destinationRelative = hostlessRuntimePath(member, classifier);
  if (destinationRelative === null) {
    continue;
  }
  const listing = execFileSync('tar', ['-tvzf', tarballPath, member], { encoding: 'utf8' }).trim();
  if (listing.startsWith('d')) {
    continue;
  }
  if (!listing.startsWith('-')) {
    throw new Error(`Unsupported runtime package entry: ${member}`);
  }
  const content = execFileSync('tar', ['-xOzf', tarballPath, member], {
    encoding: null,
    maxBuffer: 512 * 1024 * 1024,
  });
  const destination = path.resolve(resourceDir, destinationRelative);
  const resourceRoot = `${path.resolve(resourceDir)}${path.sep}`;
  if (!destination.startsWith(resourceRoot)) {
    throw new Error(`Runtime package entry escapes staging directory: ${member}`);
  }
  fs.mkdirSync(path.dirname(destination), { recursive: true });
  fs.writeFileSync(destination, content);
  const mode = listing.slice(0, 10).includes('x') ? 0o755 : 0o644;
  fs.chmodSync(destination, mode);
  inventory.push(`${mode.toString(8)}\t${destinationRelative.split(path.sep).join('/')}`);
}
inventory.sort();
fs.writeFileSync(inventoryPath, `${inventory.join('\n')}\n`);

fs.rmSync(tarballPath, { force: true });

if (!fs.existsSync(runtimePath) || !fs.existsSync(wrapperPath)) {
  throw new Error(`Package ${packageName}@${version} is missing the runtime wrapper pair`);
}
fs.writeFileSync(platformPropertiesPath, expectedPlatformProperties);
const treeDigest = digestTree(resourceDir);
fs.writeFileSync(stampPath, `${stagingSchema}\n${version}\n${integrity}\n${treeDigest}\n`);

console.log(`Staged ${runtimePath}`);

function hostlessRuntimePath(packageRelative, platform) {
  if (packageRelative.includes('\\')) {
    return null;
  }
  const parts = packageRelative.split('/');
  if (parts[0] !== 'package' || parts.some((part) => !part || part === '..')) {
    return null;
  }
  parts.shift();
  const topLevel = parts[0];
  const fileName = parts.at(-1);
  if (
    excludedTopLevel.has(topLevel) ||
    (topLevel.startsWith('tree-sitter') && topLevel.endsWith('.wasm')) ||
    (topLevel.startsWith('voice-') && topLevel.endsWith('.js')) ||
    fileName === 'cli-native.node' ||
    parts.includes('mediaremote-adapter') ||
    fileName.startsWith('copilot-runtime-bin')
  ) {
    return null;
  }
  if (topLevel === 'prebuilds') {
    if (parts[1] !== platform || parts.length < 3) {
      return null;
    }
    return path.join(...parts.slice(2));
  }
  return path.join(...parts);
}

function walkFiles(directory) {
  const files = [];
  for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
    const entryPath = path.join(directory, entry.name);
    if (entry.isDirectory()) {
      files.push(...walkFiles(entryPath));
    } else if (entry.isFile()) {
      files.push(entryPath);
    }
  }
  return files;
}

function digestTree(directory) {
  const hash = createHash('sha512');
  for (const file of walkFiles(directory).sort()) {
    const relative = path.relative(directory, file).split(path.sep).join('/');
    hash.update(relative).update('\0').update(fs.readFileSync(file)).update('\0');
  }
  return `sha512-${hash.digest('base64')}`;
}