Skip to content

Instantly share code, notes, and snippets.

@Hobart2967

Hobart2967/merge-repos.js

Last active November 2, 2022 11:47
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Hobart2967/b0dd75b8d80a0c88c0fc6ee206640b73 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Hobart2967/b0dd75b8d80a0c88c0fc6ee206640b73 to your computer and use it in GitHub Desktop.
Download ZIP
Merging packages and repos into a new, single one.
Raw
merge-repos.js
/*
To use this script, adjust the `config` object.
- set owner
- set source repos to migrate from
- set target repository name
- set target repository folder path
IMPORTANT: .npmrc needs to be configured, being to access GitHub Packages.
Execute using:
export GITHUB_TOKEN=<personal_access_token>
export GITHUB_USER=<github_user_name>
node merge-repos.js
Wait and cheer :)
*/
const os = require('os');
const fs = require('fs');
const path = require('path');
const rimraf = require('rimraf');
const child_process = require('child_process');
const axios = require('axios').default;
const util = require('util');
const stream = require('stream');
const config = {
owner: "your-github-username",
publish: false, // Set this to true when willing to delete old and create new packages (move)
repos: [
"repo-a",
"repo-b"
],
targetRepo: 'new-repo',
target: '/disk/path/for/new/repo',
};
const packages = {};
async function main() {
if (!process.env.GITHUB_TOKEN) {
console.log("Missing GITHUB_TOKEN environment variable. Exiting.");
return;
}
if (!process.env.GITHUB_USER) {
console.log("Missing GITHUB_USER environment variable. Exiting.");
return;
}
await initializeTargetRepo();
const temporaryRepoDir = createTemporaryDirectory();
const temporaryPackageDir = createTemporaryDirectory();
fs.mkdirSync(temporaryRepoDir, { recursive: true });
await addSourceRepos(temporaryRepoDir);
await publishRepo();
await movePackages(temporaryPackageDir);
await cleanup(temporaryRepoDir, temporaryPackageDir);
}
async function publishRepo() {
if (!config.publish) {
console.log("-> [SKIPPED] Publishing Target repo");
return;
}
console.log("-> Publishing Target repo");
await execute('gh', 'repo', 'create', '-y', '--private', `${config.owner}/${config.targetRepo}`, {
cwd: config.target
});
await execute('git', 'push', '-u', 'origin', 'main', {
cwd: config.target
});
}
function createTemporaryDirectory() {
return fs.mkdtempSync(os.tmpdir());
}
async function movePackages(tempWorkspace) {
const packageList = await getPackageList(config.owner);
for (const repo of config.repos) {
console.log("-> Downloading packages and versions of repo " + repo);
await downloadPackages(tempWorkspace, repo, packageList);
}
if (!config.publish) {
return;
}
await preparePackageMigrations(tempWorkspace);
if (!config.publish) {
console.log("[SKIPPED] Moving packages and versions.");
return;
}
await deletePackages();
await uploadPackages(tempWorkspace);
}
async function preparePackageMigrations(tempWorkspace) {
for (const package in packages) {
await preparePackageMigration(tempWorkspace, packages[package]);
}
}
async function uploadPackages(tempWorkspace) {
for (const package in packages) {
await uploadPackage(tempWorkspace, packages[package]);
}
}
async function uploadPackage(tempWorkspace, package) {
for (const version of package.versions) {
await execute('npm', 'publish', {
cwd: path.join(tempWorkspace, package.name, version.name)
});
}
}
async function deletePackages() {
for (const package in packages) {
console.log(`\n--> Deleting package ${package}`)
await deletePackage(packages[package]);
}
}
async function deletePackage(package) {
for (const version of package.versions) {
try {
console.log(`----> Deleting package version ${package.name}@${version.name}`)
await axios.delete(`https://api.github.com/orgs/${package.owner}/packages/npm/${package.name}/versions/${version.id}`, {
headers: getAuthHeaders()
})
} catch (err) {
console.log(err);
throw err;
}
}
}
async function getPackageList(owner) {
const url = `https://api.github.com/orgs/${owner}/packages?package_type=npm`;
const headers = getAuthHeaders();
let packages = null;
let packageList = [];
let page = 1;
while (packages === null || packages.length) {
try {
const response = await axios.get(`${url}&page=${page}`, {
headers
});
packages = response.data;
packageList.push(...packages);
page++;
} catch (err) {
console.log(err);
throw err;
}
}
return packageList;
}
async function downloadPackages(tempWorkspace, repo, packageList) {
const repoPackages = packageList.filter(x => x.repository?.name === repo);
for (const package of repoPackages) {
fs.mkdirSync(path.join(tempWorkspace, package.name));
await downloadPackageVersions(tempWorkspace, package);
}
}
async function getPackageVersionList(package) {
try {
const url = `https://api.github.com/orgs/${package.repository.owner.login}/packages/npm/${package.name}/versions`
return (await axios.get(url, { headers: getAuthHeaders() })).data;
} catch(error) {
console.log(error);
throw error;
}
}
async function downloadPackageVersions(tempWorkspace, package) {
console.log(`--> Downloading versions of ${package.name}`)
const packageVersionList = await getPackageVersionList(package);
for (const version of packageVersionList) {
fs.mkdirSync(path.join(tempWorkspace, package.name, version.name));
await downloadPackageVersion(tempWorkspace, package, version);
packages[package.name] = packages[package.name] || {
name: package.name,
repo: package.repository.name,
owner: package.owner.login,
versions: []
};
packages[package.name].versions.push({
id: version.id,
name: version.name
});
}
}
function getAuthHeaders() {
const token = Buffer
.from(`${process.env.GITHUB_USER}:${process.env.GITHUB_TOKEN}`)
.toString('base64');
return {
'Authorization': `Basic ${token}`
};
}
async function downloadPackageVersion(tempWorkspace, package, version) {
const url = (await execute('npm', 'view', `@${package.owner.login}/${package.name}@${version.name}`, 'dist.tarball')).replace(/\n/, '');
const result = await axios.get(url, { headers: getAuthHeaders(), responseType: 'stream' });
console.log('----> Downloading version ' + version.name)
const tarPath = path.join(tempWorkspace, package.name, `${version.name}.tar.gz`);
const fileStream = fs.createWriteStream(tarPath);
await new Promise((resolve, reject) => {
fileStream.on('finish', resolve);
fileStream.on('error', reject);
result.data.pipe(fileStream);
});
const extractDir = path.join(tempWorkspace, package.name, version.name);
await execute('tar', '-xzf', tarPath, '-C', extractDir);
await moveFiles(path.join(extractDir, 'package'), extractDir);
}
async function moveFiles(srcDir, destDir) {
const [move, readdir] = [
fs.rename,
fs.readdir].map(method => util.promisify(method));
const files = await readdir(srcDir);
await Promise.all(files.map((file) =>
move(
path.join(srcDir, file),
path.join(destDir, file))));
rimraf.sync(srcDir);
}
async function preparePackageMigration(tempWorkspace, package) {
for (const version of package.versions) {
console.log(`----> Migrating version ${version.name} of ${package.name}`)
const packageJsonPath = path.join(tempWorkspace, package.name, version.name, 'package.json');
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString());
packageJson.repository = {
url: `git@github.com:${config.owner}/${config.targetRepo}.git`
};
fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));
console.log(`----> ... Updated ${packageJsonPath}`)
}
}
async function initializeTargetRepo() {
console.log("-> Creating empty repository");
if (fs.existsSync(config.target)) {
rimraf.sync(config.target);
}
fs.mkdirSync(config.target, { recursive: true });
await execute('git', 'init', {
cwd: config.target
});
await execute('touch', '.gitignore', {
cwd: config.target
});
await execute('git', 'add', '.gitignore', {
cwd: config.target
});
await execute('git', 'commit', '-am', 'Initial commit', {
cwd: config.target
});
fs.mkdirSync(path.join(config.target, 'packages'), { recursive: true });
}
async function addSourceRepos(tempWorkspace) {
for (const repo of config.repos) {
console.log(`-> Adding repository '${config.owner}/${repo}'`);
await addRepo(config.owner, repo, tempWorkspace);
}
}
async function execute(...args) {
const command = args.shift();
let options;
if (typeof args[args.length - 1] !== 'string') {
options = args.pop();
}
let commandArgs = args;
return new Promise((resolve, reject) => {
let response = '';
let error = '';
const child = child_process.spawn(command , commandArgs, options);
child.stdout.on('data', (data) => response += data);
child.stderr.on('data', (data) => error += data);
child.on('close', (code) => {
if (code !== 0) {
reject(new Error(['Code ' + code, error, response].join('\n')));
return;
}
resolve(response);
});
});
}
async function addRepo(repoOwner, repoName, tempWorkspace) {
await execute('git', 'clone', `git@github.com:${repoOwner}/${repoName}`, `${tempWorkspace}/${repoName}`, {
cwd: config.target
})
await execute('git', 'remote', 'add', repoName, `${tempWorkspace}/${repoName}`, {
cwd: config.target
})
await execute('git', 'fetch', repoName, {
cwd: config.target
})
const mainBranch = (await execute('sh', '-c', 'git symbolic-ref refs/remotes/origin/HEAD | sed \'s@^refs/remotes/origin/@@\'', {
cwd: `${tempWorkspace}/${repoName}`
})).replace(/\n/, '');
await execute('git', 'subtree', 'add', '-P', `packages/${repoName}`, `git@github.com:${repoOwner}/${repoName}`, `${mainBranch}`, {
cwd: config.target
});
}
function cleanup(tempWorkspace) {
console.log("Cleaning up");
rimraf.sync(tempWorkspace);
}
main();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment