how do I ensure two asynchronous operations add to a file sequentially?

Question

I have this code:

const fs = require("fs");

const saveFile = (fileName, data) => {
  return new Promise((resolve) => {
    fs.writeFile(fileName, data, (err) => {
      resolve(true);
    });
  });
};

const readFile = (fileName) => {
  return new Promise((resolve) => {
    fs.readFile(fileName, "utf8", (err, data) => {
      resolve(data);
    });
  });
};

const filename = "test.txt";

saveFile(filename, "first");

readFile(filename).then((contents) => {
  saveFile(filename, contents + " second");
});

readFile(filename).then((contents) => {
  saveFile(filename, contents + " third");
});

I'm hoping to obtain in 'test.txt'

first second third

but instead, I get

first thirdd

The idea is that every time I receive a certain post request. I have to add more text to the file

Does someone have any solution for this?

Thank you so much!

Edit:

The problem of using async await or a chain of .then( ) is that I have to add more text to the file every time I receive a certain post request. So I don't have control over what is written or when. The Idea is that everything is written and nothing is overwritten even if two post requests are received at the same time.

I'm going to share the solution with a linked list I came up with yesterday. But I still want to know if someone has a better solution.

const saveFile = (fileName, data) => {
  return new Promise((resolve) => {
    fs.writeFile(fileName, data, (err) => {
      resolve(true);
    });
  });
};

const readFile = (fileName) => {
  return new Promise((resolve) => {
    fs.readFile(fileName, "utf8", (err, data) => {
      resolve(data);
    });
  });
};

class LinkedCommands {
  constructor(head = null) {
    this.head = head;
  }

  getLast() {
    let lastNode = this.head;
    if (lastNode) {
      while (lastNode.next) {
        lastNode = lastNode.next;
      }
    }
    return lastNode;
  }

  addCommand(command, description) {
    let lastNode = this.getLast();
    const newNode = new CommandNode(command, description);
    if (lastNode) {
      return (lastNode.next = newNode);
    }
    this.head = newNode;
    this.startCommandChain();
  }

  startCommandChain() {
    if (!this.head) return;
    this.head
      .command()
      .then(() => {
        this.pop();
        this.startCommandChain();
      })
      .catch((e) => {
        console.log("Error in linked command\n", e);
        console.log("command description:", this.head.description);
        throw e;
      });
  }

  pop() {
    if (!this.head) return;
    this.head = this.head.next;
  }
}

class CommandNode {
  constructor(command, description = null) {
    this.command = command;
    this.description = description;
    this.next = null;
  }
}

const linkedCommands = new LinkedCommands();

const filename = "test.txt";

linkedCommands.addCommand(() => saveFile(filename, "first"));

linkedCommands.addCommand(() =>
  readFile(filename).then((contents) =>
    saveFile(filename, contents + " second")
  )
);

linkedCommands.addCommand(() =>
  readFile(filename).then((contents) => saveFile(filename, contents + " third"))
);

Answer 1

Because these are async functions they notify you that the work is completed in the then function.
That means you want to use a then chain (or an async function) like so:

readFile(filename).then((contents) => {
  return saveFile(filename, contents + " second");
}).then(() => {
  return readFile(filename)
}).then((contents) => {
  saveFile(filename, contents + " third");
});

Answer 2

You can use a FIFO queue of functions that return promises for this.

const { readFile, writeFile } = require("fs/promises");

let queue = [];
let lock = false;

async function flush() {
  lock = true;

  let promise;
  do {
    promise = queue.shift();
    if (promise) await promise();
  } while (promise);

  lock = false;
}

function createAppendPromise(filename, segment) {
  return async function append() {
    const contents = await readFile(filename, "utf-8");
    await writeFile(
      filename,
      [contents.toString("utf-8"), segment].filter((s) => s).join(" ")
    );
  };
}

async function sequentialWrite(filename, segment) {
  queue.push(createAppendPromise(filename, segment));
  if (!lock) await flush();
}

async function start() {
  const filename = "test.txt";

  // Create all three promises right away
  await Promise.all(
    ["first", "second", "third"].map((segment) =>
      sequentialWrite(filename, segment)
    )
  );
}

start();

So how's this work? Well, we use a FIFO queue of promise functions. As requests come in we create them and add them to the queue.

Every time we add to the array we attempt to flush it. If there's a lock in place, we know we're already flushing, so just leave.

The flushing mechanism will grab the first function in the queue, delete it from the queue, invoke it, and await on the promise that returns. It will continue to do this until the queue is empty. Because all of this is happening asynchronously, the queue can continue to get populated while flushing.

Please keep in mind that if this file is shared on a server somewhere and you have multiple processes reading from this file (such as with horizontal scaling) you will lose data. You should instead use some kind of distributed mutex somewhere. A popular way of doing this is using Redis and redlock.

Hope this helps!

Edit: by the way, if you want to prove that this indeed works, you can add a completely random setTimeout to the createAppendPromise function.

function createAppendPromise(filename, segment) {
  const randomTime = () =>
    new Promise((resolve) => setTimeout(resolve, Math.random() * 1000));

  return async function append() {
    await randomTime();
    const contents = await readFile(filename, "utf-8");
    await writeFile(
      filename,
      [contents.toString("utf-8"), segment].filter((s) => s).join(" ")
    );
  };
}

Answer 3

Chaining is fine, even if you don't know in advance when or how many promises will be created. Just keep the end of the chain handy, and chain to it whenever you create a promise...

// this context must persist across posts
let appendChain = Promise.resolve();
const filename = "test.txt";

// assuming op's readFile and saveFile work...
const appendToFile = (filename, data) =>
  return readFile(filename).then(contents => {
    return saveFile(filename, contents + data);
  });
}

function processNewPost(data) {
  return appendChain = appendChain.then(() => {
    return appendToFile(filename, data);
  });
}

Here's a demonstration. The async functions are pretend read, write and append. The <p> tag is the simulated contents of a file. Press the button to add new data to the pretend file.

The button is for you to simulate the external event that triggers the need to append. The append function has a 1s delay, so, if you want, you can get in several button clicks before all the appends on the promise chain are write is done.

function pretendReadFile() {
  return new Promise(resolve => {
    const theFile = document.getElementById('the-file');
    resolve(theFile.innerText);
  })
}

function pretendWriteFile(data) {
  return new Promise(resolve => {
    const theFile = document.getElementById('the-file');
    theFile.innerText = data;
    resolve();
  })
}

function pretendDelay(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

function appendFile(data) {
  return pretendDelay(1000).then(() => {
    return pretendReadFile()
  }).then(result => {
    return pretendWriteFile(result + data);
  });
}

document.getElementById("my-button").addEventListener("click", () => click());

let chain = Promise.resolve();
let count = 0
function click() {
  chain = chain.then(() => appendFile(` ${count++}`));
}

<button id="my-button">Click Fast and At Random Intervals</button>
<h3>The contents of the pretend file:</h3>
<p id="the-file">empty</p>