Skip to content

Instantly share code, notes, and snippets.

@Greenheart

Greenheart/idempotent-update-log.ts

Created May 6, 2026 09:25
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Greenheart/4235359671d1eaa65057d70ab5c20f69 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Greenheart/4235359671d1eaa65057d70ab5c20f69 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
idempotent-update-log.ts
/**
* This script uses a log file to make operations idempotent, so they only can run once.
* Very useful for API calls that should not be repeated.
*
* This approach allows re-running the bulk update script without affecting already updated items.
*/
import { createWriteStream, readFileSync, existsSync } from "node:fs";
import { resolve } from "node:path";
const LOG_FILE = resolve("./updated-issues.txt");
/**
* Parse a log file from previous runs to make sure issues are only updated once.
*
* ## Format of the log file
*
* Each line contains the type of entry,
* and the GitLab `iid` for the issue or note (comment):
*
* [type] [iid]
*
* ```txt
* issue 221
* note 35
* note 49
* issue 164
* note 12
* ```
*
* By only adding entries to this file after they have been successfully updated, we can
* run the script multiple times without overwriting anything that has already been updated.
*
* @param path Path to the log file.
* @returns Two sets with the already updated issues and notes.
*/
function parseAlreadyUpdatedFromLogFile(path: string) {
const log = existsSync(path) ? readFileSync(path, "utf-8").trim() : "";
const alreadyUpdated = {
issues: new Set<number>(),
notes: new Set<number>(),
};
if (log.length) {
for (const line of log.split("\n")) {
const entry = line.split(" ");
if (entry.length !== 2) {
throw new Error("Invalid log entry: " + entry);
}
const [type, iid] = entry;
if (type === "issue") {
alreadyUpdated.issues.add(parseInt(iid, 10));
} else if (type === "note") {
alreadyUpdated.notes.add(parseInt(iid, 10));
} else {
throw new Error("Invalid log entry type: " + type);
}
}
}
return alreadyUpdated;
}
/**
* Read the previously updated issues from the log file.
* This ensures we only update each issue once.
*/
const alreadyUpdated = parseAlreadyUpdatedFromLogFile(LOG_FILE);
// Open the log file with append-mode ("a"),
// create it if it doesn't already exist (by using "a+")
const logStream = createWriteStream(LOG_FILE, { flags: "a+" });
console.log("Already updated from previous runs:");
console.dir(alreadyUpdated);
/** Realistic fake data */
const issuesToUpdate = [
{
iid: 57,
notes: [
{
iid: 1838,
},
{
iid: 4950,
},
{
iid: 3832,
},
{
iid: 4184,
},
{
iid: 4756,
},
],
},
{
iid: 46,
notes: [
{
iid: 1783,
},
{
iid: 3241,
},
{
iid: 555,
},
{
iid: 3261,
},
{
iid: 2352,
},
],
},
{
iid: 43,
notes: [
{
iid: 4170,
},
{
iid: 3605,
},
{
iid: 2181,
},
{
iid: 1652,
},
{
iid: 4492,
},
],
},
{
iid: 39,
notes: [
{
iid: 1097,
},
{
iid: 4494,
},
{
iid: 4337,
},
{
iid: 3830,
},
{
iid: 4124,
},
],
},
{
iid: 23,
notes: [
{
iid: 4289,
},
{
iid: 4970,
},
{
iid: 4417,
},
{
iid: 1176,
},
{
iid: 1403,
},
],
},
{
iid: 17,
notes: [
{
iid: 851,
},
{
iid: 109,
},
{
iid: 441,
},
{
iid: 1344,
},
{
iid: 1979,
},
],
},
{
iid: 16,
notes: [
{
iid: 2324,
},
{
iid: 3726,
},
{
iid: 2526,
},
{
iid: 3623,
},
{
iid: 2250,
},
],
},
{
iid: 15,
notes: [
{
iid: 377,
},
{
iid: 852,
},
{
iid: 4313,
},
{
iid: 1053,
},
{
iid: 3768,
},
],
},
{
iid: 14,
notes: [
{
iid: 3081,
},
{
iid: 3044,
},
{
iid: 4253,
},
{
iid: 2324,
},
{
iid: 4157,
},
],
},
{
iid: 13,
notes: [
{
iid: 1943,
},
{
iid: 2008,
},
{
iid: 3257,
},
{
iid: 590,
},
{
iid: 3089,
},
],
},
{
iid: 12,
notes: [
{
iid: 3114,
},
{
iid: 711,
},
{
iid: 1723,
},
{
iid: 2524,
},
{
iid: 4775,
},
],
},
{
iid: 11,
notes: [
{
iid: 1926,
},
{
iid: 2421,
},
{
iid: 3630,
},
{
iid: 4918,
},
{
iid: 4466,
},
],
},
{
iid: 9,
notes: [
{
iid: 3970,
},
{
iid: 2847,
},
{
iid: 4216,
},
{
iid: 3994,
},
{
iid: 3436,
},
],
},
{
iid: 8,
notes: [
{
iid: 1791,
},
{
iid: 1928,
},
{
iid: 2056,
},
{
iid: 3180,
},
{
iid: 2105,
},
],
},
{
iid: 7,
notes: [
{
iid: 4379,
},
{
iid: 3571,
},
{
iid: 3734,
},
{
iid: 2120,
},
{
iid: 810,
},
],
},
{
iid: 6,
notes: [
{
iid: 4605,
},
{
iid: 45,
},
{
iid: 868,
},
{
iid: 2353,
},
{
iid: 66,
},
],
},
{
iid: 5,
notes: [
{
iid: 2400,
},
{
iid: 198,
},
{
iid: 3771,
},
{
iid: 4033,
},
{
iid: 498,
},
],
},
{
iid: 4,
notes: [
{
iid: 1261,
},
{
iid: 2032,
},
{
iid: 4373,
},
{
iid: 2778,
},
{
iid: 2262,
},
],
},
{
iid: 3,
notes: [
{
iid: 3121,
},
{
iid: 2506,
},
{
iid: 2317,
},
{
iid: 540,
},
{
iid: 1249,
},
],
},
{
iid: 2,
notes: [
{
iid: 3814,
},
{
iid: 954,
},
{
iid: 4441,
},
{
iid: 4755,
},
{
iid: 379,
},
],
},
{
iid: 1,
notes: [
{
iid: 3243,
},
{
iid: 650,
},
{
iid: 3103,
},
{
iid: 2622,
},
{
iid: 4957,
},
],
},
];
/** Simulate delays (network requests) */
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function randomInt(min: number, max: number) {
return Math.random() * (max - min) + min;
}
function appendToLog(type: "issue" | "note", iid: number) {
logStream.write(`${type} ${iid}\n`);
}
async function updateIssue(iid: number) {
console.log("Updating issue", iid);
await sleep(randomInt(1, 3) * 1000);
const success = Math.random() < 0.5;
if (success) {
appendToLog("issue", iid);
} else {
throw new Error(`Failed to update issue ${iid} due to a simulated NetworkError`);
}
}
async function updateNote(iid: number) {
console.log("Updating note", iid);
await sleep(randomInt(1, 2) * 1000);
const success = Math.random() < 0.95;
if (success) {
appendToLog("note", iid);
} else {
throw new Error(`Failed to update note ${iid} due to a simulated NetworkError`);
}
}
async function updateAllIssues() {
try {
for (const issue of issuesToUpdate) {
// Skip updating issues if they exist in the log file.
if (!alreadyUpdated.issues.has(issue.iid)) {
await updateIssue(issue.iid);
} else {
console.log("Skipping issue", issue.iid, "because it was already updated");
}
for (const note of issue.notes) {
// Skip updating notes if they exist in the log file.
if (!alreadyUpdated.notes.has(note.iid)) {
await updateNote(note.iid);
} else {
console.log("Skipping note", note.iid, "because it was already updated");
}
}
}
} catch (error) {
// Catch all errors to be able to exit gracefully
console.error("Aborting due to error:", error);
} finally {
logStream.end();
}
}
async function main() {
await updateAllIssues();
}
await main();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment